Ankiのcode reading
AnkiのメインコードはPythonだが、少し前からDBまわりがRustで置き換えられ始めていた project initialization
事前準備
protocへのpathをPROTOCに通しておく必要もある python, typescript, rustで使う共通の型定義をこの規格でproto/に書いている
buildはrslib/protoのanki_proto crateが実行する
export_apkgに必要な型定義の出力
code:buf.gen.yaml
version: v2
clean: true
inputs:
- directory: ../proto
paths:
- ../proto/anki/cards.proto
- ../proto/anki/decks.proto
- ../proto/anki/notes.proto
- ../proto/anki/notetypes.proto
plugins:
- local: protoc-gen-es
opt:
- target=ts
- import_extension=ts # v2ではimport_extension=.ts
#- ts_nocheck=false # v1で必要 out: ./gen
code:deno.json
{
"imports": {
"@bufbuild/protobuf": "npm:@bufbuild/protobuf@2"
},
"nodeModulesDir": true,
"tasks": {
"buf": "deno cache --allow-scripts=npm:@bufbuild/buf npm:@bufbuild/buf@1 npm:@bufbuild/protoc-gen-es@2 && PATH=$PATH:./node_modules/.bin deno run --allow-env --allow-read --allow-write --allow-run npm:@bufbuild/buf@1/buf"
}
}
code:mod.ts
export {
type Deck,
type Deck_Common,
Deck_CommonSchema,
type Deck_Filtered,
type Deck_Filtered_SearchTerm,
type Deck_KindContainer,
Deck_KindContainerSchema,
type Deck_Normal,
type Deck_Normal_DayLimit,
} from "./gen/anki/decks_pb.ts";
export type { Card, FsrsMemoryState } from "./gen/anki/cards_pb.ts";
export type { Note } from "./gen/anki/notes_pb.ts";
export {
type Notetype,
type Notetype_Config,
type Notetype_Config_CardRequirement,
type Notetype_Config_CardRequirement_Kind,
type Notetype_Config_Kind,
Notetype_ConfigSchema,
type Notetype_Field,
type Notetype_Field_Config,
Notetype_Field_ConfigSchema,
type Notetype_Template,
type Notetype_Template_Config,
Notetype_Template_ConfigSchema,
type StockNotetype_OriginalStockKind,
} from "./gen/anki/notetypes_pb.ts";
export type { UInt32 } from "./gen/anki/generic_pb.ts";
rootで↓を実行すると、Deck,Note,Card,Notetypeが出力される
v1だとclass実装
bundle size: 64.0kB
v2だとfunction baseだが、fileDescがbase64でバイナリデータを埋め込んでいる
bundle size: 77.6kB
うーん、似たりよったりだ
10kBなら、どちらをとっても対して変わらないか……
code reading
export
import_export::package::apkg::export::Collection::export_apkgでapkgを作る https://code2svg.vercel.app/svg/L24-66/https://raw.githubusercontent.com/ankitects/anki/24.06.2/rslib/src/import_export/package/apkg/export.rs#.svg https://github.com/ankitects/anki/blob/24.06.2/rslib/src/import_export/package/apkg/export.rs#L24-L66
legacy optionを使わなければ、Version::Latestやcollection.anki21bが使われる
Collection::export_into_collection_fileでsqlite DBを作る
Collection::gather_notes
NoteTableGuard::Collection::gather_cards
CardTableGuard::Collection::gather_decks
scheduleがいらないときは、ExchangeData::reset_cards_and_notesする
deck configもいれるときはCardTableGuard::Collection::gather_deck_configsを実行する
最後にExchangeData::reset_decksとExchangeData::check_idsをしたらおわり
code:rust
pub(super) fn insert_data(&mut self, data: &ExchangeData) -> Result<()> {
self.transact_no_undo(|col| {
col.insert_decks(&data.decks)?;
col.insert_notes(&data.notes)?;
col.insert_cards(&data.cards)?;
col.insert_notetypes(&data.notetypes)?;
col.insert_revlog(&data.revlog)?;
col.insert_deck_configs(&data.deck_configs)
})
}
decks,notes,cards,notetypes,deck_configsさえあればよさそう
集めたapkgデータをimport_export::package::colpkg::export::export_collectionでzipして.apkgにする
import_export::package::colpkg::export::export_collection::write_collection()とimport_export::package::colpkg::export::export_collection::write_media_map()で使っている
コードの共通化が目的
Protobuf files defining the interface the frontend and backend components use to talk to each other, and how Anki stores some of the data inside its SQLite database. These files are used to generate Rust, Python and TypeScript bindings.
Rustコードにはmethodsなどのimplだけが定義されている
import
こっちも調べる
❌️もしimportでnoteからcardの自動生成をやっているなら、deno-ankiでcardを自前生成する必要がなくなる 自動生成してなかった
ExchangeData::gather_from_archiveでExchangeDataを.apkgから生成
新規生成はせず、apkgから読み出したExchangeData.cardsを使っているみたい
カードの上書きはできない？
テキスト系ファイルからのimport
Collection::import_csvやCollection::import_jsonでimportする
sqliteの初期化
関数内でpragmaの設定とfunctionを追加している
新規作成の手順
1. pragmaとfunctionの設定
in anki::storage::sqlite::open_or_create_collection_db
code:pragma.rs
db.pragma_update(None, "locking_mode", "exclusive")?;
db.pragma_update(None, "page_size", 4096)?;
db.pragma_update(None, "cache_size", -40 * 1024)?;
db.pragma_update(None, "legacy_file_format", false)?;
db.pragma_update(None, "journal_mode", "wal")?;
SQL文の中にPRAGMA locking_mode = exclusiveなどとも書ける
code:function.rs
add_field_index_function(&db)?;
add_regexp_function(&db)?;
add_regexp_fields_function(&db)?;
add_regexp_tags_function(&db)?;
add_without_combining_function(&db)?;
add_fnvhash_function(&db)?;
add_extract_custom_data_function(&db)?;
add_extract_fsrs_variable(&db)?;
add_extract_fsrs_retrievability(&db)?;
add_extract_fsrs_relative_overdueness(&db)?;
db.create_collation("unicase", unicase_compare)?;
2. execute begin exclusive
3. execute schema11.sql
4. execute update col set crt=?, scm=?, ver=?, conf=?
scmはDate.now()
verは11
最終的に18になる
confにはanki::config::schema11::schema11_config_as_stringで作ったJSON stringが入る
offsetを引数にとる
あとでstorage.upgrade_config_to_schema14()で削除される
5. upgrade to 18
code:rs
storage.db.execute_batch(include_str!("schema14_upgrade.sql"))?;
colにあったdconfをJSONにparseし、deck_configにいれる
初期設定時はstorage.add_default_deck_config()で値を作る(後述)
code:rs
storage.upgrade_deck_conf_to_schema14()?;
storage.upgrade_tags_to_schema14()?;
storage.upgrade_config_to_schema14()?;でcolテーブルのconfに入れたJSONをconfテーブルに展開する
key: JSONのkey
val: JSONのvalue
usn: 0
mtime_specs: 0
usnとmtime_specsはkey: curModelだけadd_stock_notetypes()で書き換えられている
code:rs
storage.upgrade_config_to_schema14()?;
storage.db.execute_batch(include_str!("schema15_upgrade.sql"))?;
storage.upgrade_notetypes_to_schema15()?;
storage.upgrade_decks_to_schema15(server)?;
storage.upgrade_deck_conf_to_schema15()?;
storage.upgrade_deck_conf_to_schema16(server)?;
storage.db.execute_batch("update col set ver = 16")?;
storage.upgrade_tags_to_schema17()?;
storage.db.execute_batch("update col set ver = 17")?;
storage.db.execute_batch(include_str!("schema18_upgrade.sql"))?;
DeckConfigを入れる
id: 1
name: Default
mtime_secs: 0
usn: 0
値に応じてidが更新されるので、受け取って保持しておく
code:rs
storage.add_default_deck_config(tr)?;
Deckを入れる
id: 1
name: DeckConfig.nameと同じ
as_native_strを通じて格納
mtime_secs: 0
usn: 0
common:
study_collapsed: true
browser_collapsed: true
ほかはdefault
kind:
config_id: 1
bigint
ほかはdefault
code:rs
storage.add_default_deck(tr)?;
code:rs
storage.add_stock_notetypes(tr)?;
storage.commit_trx()?; // commit
なかなかややこしいことになっている
一つにまとめたいところ
code:schema18.sql
CREATE TABLE cards (
id integer PRIMARY KEY,
nid integer NOT NULL,
did integer NOT NULL,
ord integer NOT NULL,
mod integer NOT NULL,
usn integer NOT NULL,
type integer NOT NULL,
queue integer NOT NULL,
due integer NOT NULL,
ivl integer NOT NULL,
factor integer NOT NULL,
reps integer NOT NULL,
lapses integer NOT NULL,
left integer NOT NULL,
odue integer NOT NULL,
odid integer NOT NULL,
flags integer NOT NULL,
data text NOT NULL
)
CREATE TABLE col (
id integer PRIMARY KEY,
crt integer NOT NULL,
mod integer NOT NULL,
scm integer NOT NULL,
ver integer NOT NULL,
dty integer NOT NULL,
usn integer NOT NULL,
ls integer NOT NULL,
conf text NOT NULL,
models text NOT NULL,
decks text NOT NULL,
dconf text NOT NULL,
tags text NOT NULL
)
CREATE TABLE config (
KEY text NOT NULL PRIMARY KEY,
usn integer NOT NULL,
mtime_secs integer NOT NULL,
val blob NOT NULL
) without rowid
CREATE TABLE deck_config (
id integer PRIMARY KEY NOT NULL,
name text NOT NULL COLLATE unicase,
mtime_secs integer NOT NULL,
usn integer NOT NULL,
config blob NOT NULL
)
CREATE TABLE decks (
id integer PRIMARY KEY NOT NULL,
name text NOT NULL COLLATE unicase,
mtime_secs integer NOT NULL,
usn integer NOT NULL,
common blob NOT NULL,
kind blob NOT NULL
)
CREATE TABLE fields (
ntid integer NOT NULL,
ord integer NOT NULL,
name text NOT NULL COLLATE unicase,
config blob NOT NULL,
PRIMARY KEY (ntid, ord)
) without rowid
CREATE TABLE graves (
oid integer NOT NULL,
type integer NOT NULL,
usn integer NOT NULL,
PRIMARY KEY (oid, type)
) WITHOUT ROWID
CREATE TABLE notes (
id integer PRIMARY KEY,
guid text NOT NULL,
mid integer NOT NULL,
mod integer NOT NULL,
usn integer NOT NULL,
tags text NOT NULL,
flds text NOT NULL,
-- The use of type integer for sfld is deliberate, because it means that integer values in this
-- field will sort numerically.
sfld integer NOT NULL,
csum integer NOT NULL,
flags integer NOT NULL,
data text NOT NULL
)
CREATE TABLE notetypes (
id integer NOT NULL PRIMARY KEY,
name text NOT NULL COLLATE unicase,
mtime_secs integer NOT NULL,
usn integer NOT NULL,
config blob NOT NULL
)
CREATE TABLE revlog (
id integer PRIMARY KEY,
cid integer NOT NULL,
usn integer NOT NULL,
ease integer NOT NULL,
ivl integer NOT NULL,
lastIvl integer NOT NULL,
factor integer NOT NULL,
time integer NOT NULL,
type integer NOT NULL
)
CREATE TABLE tags (
tag text NOT NULL PRIMARY KEY COLLATE unicase,
usn integer NOT NULL,
collapsed boolean NOT NULL,
config blob NULL
) without rowid
CREATE TABLE templates (
ntid integer NOT NULL,
ord integer NOT NULL,
name text NOT NULL COLLATE unicase,
mtime_secs integer NOT NULL,
usn integer NOT NULL,
config blob NOT NULL,
PRIMARY KEY (ntid, ord)
) without rowid
CREATE INDEX idx_cards_odid ON cards (odid)
WHERE odid != 0
CREATE UNIQUE INDEX idx_decks_name ON decks (name)
CREATE UNIQUE INDEX idx_fields_name_ntid ON fields (name, ntid)
CREATE INDEX idx_graves_pending ON graves (usn)
CREATE INDEX idx_notes_mid ON notes (mid)
CREATE UNIQUE INDEX idx_notetypes_name ON notetypes (name)
CREATE INDEX idx_notetypes_usn ON notetypes (usn)
CREATE UNIQUE INDEX idx_templates_name_ntid ON templates (name, ntid)
CREATE INDEX idx_templates_usn ON templates (usn)
CREATE INDEX ix_cards_nid ON cards (nid)
CREATE INDEX ix_cards_sched ON cards (did, queue, due)
CREATE INDEX ix_cards_usn ON cards (usn)
CREATE INDEX ix_notes_csum ON notes (csum)
CREATE INDEX ix_notes_usn ON notes (usn)
CREATE INDEX ix_revlog_cid ON revlog (cid)
CREATE INDEX ix_revlog_usn ON revlog (usn)
Collection::new_minimalではDELETE FROM notetypesも実行する
notetypesに含まれるすべてのrecordを削除する
NoteからCardを作る
Collection::generate_cards_for_existing_note
anki::import_export::text::import::ForeignData::importで呼び出しているanki::import_export::text::import::Context::generate_missing_cardsで使われている
他にはanki::notes::Collection::update_note_inner_generating_cardsでも使われている
上2つは内部でCollection::generate_cards_for_noteを呼び出している
anki::notetype::cardgen::CardGenContext::new_cards_requiredで新規作成する必要があるcardsのみ選び出し、anki::notetype::cardgen::Collection::add_generated_cardsで作成したcardsを追加する
media filesのexport
anki::import_export::gather::ExchangeData::gather_media_namesでExchangeData::media_filenamesにNote(anki::import_export::gather::gather_media_names_from_note)やNotetype(anki::notetype::Notetype::gather_media_names)から得たfilenamesを集める
media filesの本体はanki::import_export::package::colpkg::export::write_mediaでzipにzstd圧縮されて格納される どうやってTSに変換するか
sql実行あたりの型から攻めていくか
anki::import_export::insert::Collection::insert_dataでDeck,Note,Card,Notetypeを直接変換してsqlite DBに格納している
よって、これらの型に合う情報を組み立てれればいい
code:rust
pub struct Deck {
pub id: DeckId, // i64
pub name: NativeDeckName, // String
pub mtime_secs: TimestampSecs, // i64
pub usn: Usn, // i32
pub common: DeckCommon,
pub kind: DeckKind, // enum { Normal, Filtered }
}
pub struct DeckCommon {
pub study_collapsed: bool,
pub browser_collapsed: bool,
pub last_day_studied: u32,
pub new_studied: i32,
pub review_studied: i32,
pub milliseconds_studied: i32,
/// previously set in the v1 scheduler,
/// but not currently used for anything
pub learning_studied: i32,
pub other: ::prost::alloc::vec::Vec<u8>,
}
pub struct Note {
pub id: NoteId, // i64
pub guid: String,
pub notetype_id: NotetypeId, // i64
pub mtime: TimestampSecs, // i64
pub usn: Usn, // i32
pub tags: Vec<String>,
fields: Vec<String>,
pub(crate) sort_field: Option<String>,
pub(crate) checksum: Option<u32>,
}
pub struct Card {
pub(crate) id: CardId, // i64
pub(crate) note_id: NoteId, // i64
pub(crate) deck_id: DeckId, // i64
pub(crate) template_idx: u16,
pub(crate) mtime: TimestampSecs, // i64
pub(crate) usn: Usn, // i32
pub(crate) ctype: CardType, // enum
pub(crate) queue: CardQueue, // enum
pub(crate) due: i32,
pub(crate) interval: u32,
pub(crate) ease_factor: u16,
pub(crate) reps: u32,
pub(crate) lapses: u32,
pub(crate) remaining_steps: u32,
pub(crate) original_due: i32,
pub(crate) original_deck_id: DeckId, // i64
pub(crate) flags: u8,
/// The position in the new queue before leaving it.
pub(crate) original_position: Option<u32>,
pub(crate) memory_state: Option<FsrsMemoryState>,
pub(crate) desired_retention: Option<f32>,
/// JSON object or empty; exposed through the reviewer for persisting custom
/// state
pub(crate) custom_data: String,
}
pub struct FsrsMemoryState {
pub stability: f32,
pub difficulty: f32,
}
pub struct Notetype {
pub id: NotetypeId, // i64
pub name: String,
pub mtime_secs: TimestampSecs, // i64
pub usn: Usn, // i32
pub fields: Vec<NoteField>,
pub templates: Vec<CardTemplate>,
pub config: NotetypeConfig,
}
pub struct NoteField {
pub ord: Option<u32>,
pub name: String,
pub config: NoteFieldConfig,
}
pub struct NoteFieldConfig {
pub sticky: bool,
pub rtl: bool,
pub font_name: ::prost::alloc::string::String,
pub font_size: u32,
pub description: ::prost::alloc::string::String,
pub plain_text: bool,
pub collapsed: bool,
pub exclude_from_search: bool,
/// used for merging notetypes on import (Anki 23.10)
pub id: ::core::option::Option<i64>,
/// Can be used to uniquely identify required fields.
pub tag: ::core::option::Option<u32>,
pub prevent_deletion: bool,
pub other: ::prost::alloc::vec::Vec<u8>,
}
pub struct CardTemplate {
pub ord: Option<u32>,
pub mtime_secs: TimestampSecs,
pub usn: Usn,
pub name: String,
pub config: CardTemplateConfig,
}
pub struct CardTemplateConfig {
pub q_format: ::prost::alloc::string::String,
pub a_format: ::prost::alloc::string::String,
pub q_format_browser: ::prost::alloc::string::String,
pub a_format_browser: ::prost::alloc::string::String,
pub target_deck_id: i64,
pub browser_font_name: ::prost::alloc::string::String,
pub browser_font_size: u32,
/// used for merging notetypes on import (Anki 23.10)
pub id: ::core::option::Option<i64>,
pub other: ::prost::alloc::vec::Vec<u8>,
}
pub struct NotetypeConfig {
pub kind: i32,
pub sort_field_idx: u32,
pub css: ::prost::alloc::string::String,
/// This is now stored separately; retrieve with DefaultsForAdding()
pub target_deck_id_unused: i64,
pub latex_pre: ::prost::alloc::string::String,
pub latex_post: ::prost::alloc::string::String,
pub latex_svg: bool,
pub reqs: ::prost::alloc::vec::Vec<config::CardRequirement>,
/// Only set on notetypes created with Anki 2.1.62+.
pub original_stock_kind: i32,
/// the id in the source collection for imported notetypes (Anki 23.10)
pub original_id: ::core::option::Option<i64>,
pub other: ::prost::alloc::vec::Vec<u8>,
}
pub struct CardRequirement {
pub card_ord: u32,
pub kind: i32,
pub field_ords: ::prost::alloc::vec::Vec<u32>,
}
Deckの書き込み
Noteの書き込み
単純
Cardの書き込み
単純
Notetypeの書き込み
だいぶ込み入っている
encodeが必要なものは上に書いたtsファイルにすべて書き出した
一時期Denoへの移行を進めていたが、やめたっぽい